# -*- coding: utf-8 -*-
import logging
import subprocess
import traceback
from io import BytesIO
from itertools import chain
from pprint import pprint

import magic
import requests
from PyPDF2.pdf import PdfFileReader
from bs4 import BeautifulSoup
from xlrd import open_workbook, XLRDError

from plugin.db.mongo_base import MongoBase
from pyspider.libs.utils import timeout

logging.basicConfig(level=logging.INFO,
                    format='[%(levelname)1.1s %(asctime)s] [%(filename)s %(funcName)s line:%(lineno)d] %(message)s')
OPEN_FILE_TIMEOUT = 30
HEADERS_CONF = {
    'FFD8FF': 'JPEG (jpg)',
    '89504E47': 'PNG (png)',
    '47494638': 'GIF (gif)',
    '49492A00': 'TIFF (tif)',
    '424D': 'Windows Bitmap (bmp)',
    '41433130': 'CAD (dwg)',
    '38425053': 'Adobe Photoshop (psd)',
    '7B5C727466': 'Rich Text Format (rtf)',
    '3C3F786D6C': 'XML (xml)',
    '68746D6C3E': 'HTML (html)',
    '44656C69766572792D646174653A': 'Email [thorough only] (eml)',
    'CFAD12FEC5FD746F': 'Outlook Express (dbx)',
    '2142444E': 'Outlook (pst)',
    'D0CF11E0': 'MS Word/Excel (xls.or.doc)',
    '5374616E64617264204A': 'MS Access (mdb)',
    'FF575043': 'WordPerfect (wpd)',
    '252150532D41646F6265': 'Postscript (eps.or.ps)',
    '255044462D312E': 'Adobe Acrobat (pdf)',
    'AC9EBD8F': 'Quicken (qdf)',
    'E3828596': 'Windows Password (pwl)',
    '504B0304': 'ZIP Archive (zip)',
    '52617221': 'RAR Archive (rar)',
    '57415645': 'Wave (wav)',
    '41564920': 'AVI (avi)',
    '2E7261FD': 'Real Audio (ram)',
    '2E524D46': 'Real Media (rm)',
    '000001BA': 'MPEG (mpg)',
    '000001B3': 'MPEG (mpg)',
    '6D6F6F76': 'Quicktime (mov)',
    '3026B2758E66CF11': 'Windows Media (asf)',
    '4D546864': 'MIDI (mid)',
}
HEADERS_INFO_CONF = {
    'D0CF11E0': [
        (8, 'Header Signature (8 bytes):区别文件类型的文件头，固定为D0, CF,11, E0, A1, B1, 1A, E1.'),
        (16, 'Header CLSID (16 bytes): 保留字节，全都是0'),
        (2, 'Minor Version (2 bytes):文件小版本号，固定为3E'),
        (2, 'Major Version (2 bytes):文件主版本号，有3个4两种值，反正这里都是3'),
        (2, 'Byte Order (2 bytes):固定为FFFE，表示Little Endian'),
        (2, 'Sector Shift (2 bytes):表示每个扇区的大小。2和这里的数字做指数运算后即为扇区的大小。只有两种值可选，9和C，即9和12，分别表示512字节和4096字节，也分别对应了版本号3和4。'),
        (2, 'Mini Sector Shift (2 bytes):固定为0006，表示Mini扇区的大小，也就是64个字节'),
        (6, 'Reserved (6 bytes):保留字，全都是0'),
        (4, 'Number of Directory Sectors (4 bytes): Directory Sectors的数量，就是目录扇区的数量。若版本号为3，则该值为0'),
        (4, 'Number of FAT Sectors (4 bytes): FAT扇区的数量'),
        (4, 'First Directory Sector Location (4 bytes):文件目录流的起始扇区编号'),
        (4, 'Transaction Signature Number (4 bytes):都是0，没啥用'),
        (4, 'Mini Stream Cutoff Size (4 bytes):固定为00001000。表示文件最小的存储大小，为4096，即4k'),
        (4, 'First Mini FAT Sector Location (4 bytes): mini FAT的起始扇区编号'),
        (4, 'Number of Mini FAT Sectors (4 bytes): mini FAT的数量'),
        (4, 'First DIFAT Sector Location (4 bytes): DIFAT的起始扇区编号'),
        (4, 'Number of DIFAT Sectors (4 bytes): DIFAT的数量'),
        (436, 'DIFAT (436 bytes):包含了109个DIFAT，每个DIFAT占用4个字节。'),
    ],
    '504B0304': [
        (4, 'Local file header signature = 0x04034b50 (read as a little-endian number): 文件头标识，值固定(0x04034b50)'),
        (2, 'Version needed to extract (minimum): 解压文件所需 pkware最低版本'),
        (2, 'General purpose bit flag: 通用比特标志位(置比特0位=加密，详情见后)'),
        (2, 'Compression method: 压缩方式（详情见后）'),
        (2, 'File last modification time: 文件最后修改时间'),
        (2, 'File last modification date: 文件最后修改日期'),
        (2, 'CRC-32: CRC-32校验码'),
        (4, 'Compressed size: 压缩后的大小'),
        (4, 'Uncompressed size: 未压缩的大小'),
        (2, 'File name length (n): 文件名长度'),
        (2, 'Extra field length (m): 扩展区长度'),
    ]
}


class CommandLineError(Exception):
    """The traceback of all CommandLineError's is supressed when the
    errors occur on the command line to provide a useful command line
    interface.
    """

    def __init__(self, command, exit_code, stdout, stderr):
        self.command = command
        self.exit_code = exit_code
        self.stdout = stdout
        self.stderr = stderr


def run_shell(args):
    pipe = subprocess.Popen(
        args,
        stdout=subprocess.PIPE, stderr=subprocess.PIPE,
    )
    stdout, stderr = pipe.communicate()

    # if pipe is busted, raise an error (unlike Fabric)
    if pipe.returncode != 0:
        raise CommandLineError(' '.join(args), pipe.returncode, stdout, stderr)

    return stdout, stderr


def bytes_to_hex(bytes_str, join=''):
    """
    convers bytes to hex
    :param bytes_str: bytes_str
    :param join: hex 间隔
    :return:
    """
    return join.join('{:0>2}'.format(hex(_bytes).lstrip('0x')) for _bytes in bytes_str).upper()


class CheckFileIntergtiry(object):
    """
    校验下载文件的完整性
    """

    def check(self, content, file_type=None):
        if len(content) < 14:
            return False
        header_type = self.judge_file_header(content)
        mime_type = magic.from_buffer(content, mime=True)
        logging.info('header_type: {}, mime_type: {}'.format(header_type, mime_type))
        if header_type:
            if 'xml' in header_type:
                try:
                    BeautifulSoup(content)
                    return True
                except:
                    traceback.print_exc()
            if 'pdf' in header_type:
                pdf_content = content.strip()
                pdf_header = content.split(b'\r', 1)[0]
                print('文件头，即，PDF文件的第一行，它用来定义PDF的版本', pdf_header, sep='\n\t')
                if pdf_content.endswith(b'%%EOF') or (
                        b'trailer' in pdf_content and b'%%EOF' in pdf_content.rsplit(b'trailer', 1)[-1]):
                    # pdf_trailer = content.rsplit(b'trailer', 1)[-1]
                    # print('文件尾，应用程序应从尾段开始阅读PDF文件。最后一行只包含%%EOF', pdf_trailer, sep='\n\t')
                    return True
        else:
            if '.pdf' == file_type.lower() and 'pdf' not in mime_type:
                return True
        if isinstance(file_type, str):
            return self.open_file(content, file_type)

    def open_pdf(self, content):
        PdfFileReader(BytesIO(content)).getNumPages()

    def check_excel(self, content):
        try:
            open_workbook(file_contents=content)
        except XLRDError as e:
            if 'Workbook is encrypted' not in e.__str__():
                raise e

    def open_xls(self, content):
        self.check_excel(content)

    def open_xlsx(self, content):
        self.check_excel(content)

    # def open_docx(self, content):
    #     try:
    #         Document(BytesIO(content))
    #     except BadZipFile as e:
    #         if 'File is not a zip file' in e.__str__():
    #             doc_f = NamedTemporaryFile(suffix='.doc')
    #             doc_f.write(content)
    #             doc_f.seek(0)
    #             run_shell(['antiword', doc_f.name])
    #
    # def open_doc(self, content):
    #     doc_f = NamedTemporaryFile(suffix='.doc')
    #     doc_f.write(content)
    #     doc_f.seek(0)
    #     try:
    #         run_shell(['antiword', doc_f.name])
    #     except CommandLineError as e:
    #         if b'It seems to be a ZIP file,' in e.stderr:
    #             Document(BytesIO(content))
    #         else:
    #             run_shell(['soffice', '--headless', '--invisible',
    #                        '--convert-to', 'docx', doc_f.name,
    #                        '--outdir', '/tmp'])
    #             Document(doc_f.name.replace('.doc', '.docx'))

    def open_file(self, content, file_type):
        """
        检测文件打开异常
        :param content:
        :param file_type:
        :return:
        """
        with timeout(OPEN_FILE_TIMEOUT, 'try open file timeout'):
            try:
                file_type = file_type.strip().strip('.').strip().lower()

                if getattr(self, 'open_{}'.format(file_type), None):
                    logging.info('try open {} file !'.format(file_type))

                    try:
                        getattr(self, 'open_{}'.format(file_type))(content)
                    except Exception as e:
                        logging.error(e)
                        return False
                else:
                    logging.warning('{} file is not support open'.format(file_type))
                    return -1
                return True
            except:
                return False

    def judge_file_header(self, content):
        """
        :param content:
        :return:
        """
        file_headers = bytes_to_hex(content[:14])
        header_type, headers_info = None, None
        for _start, _type in HEADERS_CONF.items():
            if file_headers.startswith(_start):
                headers_info = HEADERS_INFO_CONF.get(_start)
                header_type = _type
                logging.debug(header_type)

        if headers_info:
            info_start = 1
            info_end = 0
            for length, header_mean in headers_info:
                logging.debug(header_mean)
                info_end += length
                info = content[info_start - 1:info_end]
                hex_info = bytes_to_hex(info, ' ')
                if '大小' in header_mean:
                    logging.debug('\t{}\n\t{}\n\t{}'.format(info, hex_info, int(hex_info.replace(' ', ''), base=16)))
                else:
                    logging.debug('\t{}\n\t{}'.format(info, hex_info))
                info_start += length
        return header_type


def test(config, env='loacl'):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.35',
        # 'Host': 'www.pyrating.cn',
        'Cookie': 'ASP.NET_SessionId=5jpx3qpnpzxgq4rrbqnp1eo0'
    }
    notie_info_list = []
    right_info = []
    bad_info = []
    current_config = config[env]
    oss_file_url = current_config['base_oss_path']
    juchao_find_res = []
    bad_case_list = current_config.get('bad_case_list', [])
    if current_config.get('juchao_find_cond'):
        find_cond = current_config['juchao_find_cond']
        uri = current_config['juchao_mongo_uri']
        db = 'cr_data'
        col = 'finance.juchao.item'
        juchao_col = MongoBase(uri, database=db, collection=col, appname='spider_{}_{}'.format(db, col)).col
        # juchao_col = MongoClient(current_config['juchao_mongo_uri'], maxIdleTimeMS=10 * 1000)['cr_data']['finance.juchao.item']
        juchao_find_res = juchao_col.find(
            find_cond,
            {'file_size': 1, 'file_url': 1, 'file_type': 1, 'oss_path': 1, 'src_id': 1, 'process_error': 1,
             'update_time': 1, 'time': 1, 'source_type': 1, 'publish': 1, 'column': 1, '_id': 0},
            sort=[
                ('file_size', 1),
                # ('update_time', 1),
                # ('_id', 1),
                # ('time', -1),
            ])
        logging.info('juchao_find_res', juchao_find_res.count())

    for notie_info in chain(juchao_find_res, bad_case_list):
        file_url = oss_file_url.format(notie_info['oss_path']) if current_config.get('check_oss') else notie_info[
            'file_url']
        pprint(notie_info)
        logging.info(file_url)
        # rsp = requests.session().request(**
        #                                  {'headers': headers,
        #                                   'method': 'POST',
        #                                   'url': 'http://www.pyrating.cn/download?path=BrLmR6ymiZcNjfUycXrOMvuQM5kPt1fRcTgQIyfChWtkJeNcHWOcvYP5xirEx8uDJqq5p2yNWnDJ8uCZ0nDp4oC3aoDYvmP6OZDJ0'})
        rsp = requests.get(file_url,
                           headers=headers,
                           timeout=(60, 120),
                           proxies={
                               'http': 'http://127.0.0.1:6666',
                               'https': 'http://127.0.0.1:6666'
                           }
                           )
        pprint(rsp.headers)
        content = rsp.content
        if current_config.get('save_file'):
            if current_config.get('check_oss'):
                file_name = 'test_oss_{}.{}'.format(notie_info.get('src_id', ''), notie_info['file_type'])
            else:
                file_name = 'test_{}.{}'.format(notie_info.get('src_id', ''), notie_info['file_type'])
            with open(file_name, 'wb') as f:
                f.write(content)
        logging.info(rsp.status_code)
        logging.info('Content-Length:{} file_size:{}'.format(rsp.headers.get('Content-Length'), len(content)))
        check_res = CheckFileIntergtiry().check(content, notie_info['file_type'])
        logging.info('check_res[{}]: {}'.format(file_url, check_res))
        if check_res:
            right_info.append(notie_info)
        else:
            bad_info.append(notie_info)
    if current_config.get('juchao_find_cond'):
        logging.info('right_info: ', len(right_info))
        pprint(list(map(lambda _d: _d['src_id'], right_info)))
        logging.info('bad_info: ', len(bad_info))
        pprint(list(map(lambda _d: _d['src_id'], bad_info)))
    logging.info('right_info: {}'.format(len(right_info)))
    logging.info('bad_info: {}'.format(len(bad_info)))


if __name__ == '__main__':
    find_cond = {
        # "src_id": "N1P015431c2996bbadcf66dce39ed9339fdcf",

        # "file_url" : "http://www3.hkexnews.hk/listedco/listconews/SEHK/2019/0329/LTN201903292295_C.pdf",
        # "source_type": "F006_5",
        # 'source_type': {'$exists': 1},
        # 'process_error': {'$in': [
        #     'LOAD_ERROR',
        #     'EXCEPTION',
        #     # 'SUCCESS'
        # ]},
        # "update_time": {'$gt': datetime(2019, 3, 25)},
        # 'file_type': '.pdf'
        # 'file_type': {'$in': [
        # '.xls',
        # '.xlsx',
        # '.doc',
        # '.docx',
        # '.pdf',
        # ]},
        # 'src_id': {'$in': [
        # ]},
    }

    bad_case_list = [
        # {   #  header_type None
        #     'file_url': 'https://www.sec.gov/Archives/edgar/data/1247015/000112760219011903/doc1.txt',
        #     'file_type': '.txt',
        # },
        # {  # pdf 结尾异常
        #     'file_url': 'http://www.neeq.com.cn/disclosure/2019/2019-03-18/1552901646_724134.pdf',
        #     'file_type': '.pdf',
        # },
        # {  # 安装 antiword
        #     'file_url': 'http://gi.mnr.gov.cn/201903/P020190315387701871542.doc',
        #     'file_type': '.doc',
        # },
        # {   #  python xlrd assert
        #     "file_url": "http://www3.hkexnews.hk/listedco/listconews/SEHK/2019/0320/LTN20190320123.xls",
        #     'file_type': '.xls',
        # },
        # {  # 加密
        #     "file_url": "http://www.hkexnews.hk/listedco/listconews/SEHK/2015/0508/LTN20150508134_C.xls",
        #     'file_type': '.xls',
        # },
        # {  # post 请求
        #     "file_url": "http://www.shclearing.com/wcm/shch/pages/client/download/download.jsp?FileName=P020190320528235660690.pdf&DownName=",
        #     'file_type': '.pdf',
        # },
        # {  # doc 文件 是 ZIP file
        #     "file_url": "http://reportdocs.static.szse.cn/UpFiles/cfwj/2012-08-10_002102163.doc",
        #     'file_type': '.doc',
        # },
        # {  # pdf 加密异常
        #     "file_url": "http://www.shclearing.com/wcm/shch/pages/client/download/download.jsp?FileName=P020190325495918336075.pdf&DownName=",
        #     'file_type': '.pdf',
        # },
        # {  # doc antiword  识别异常？？？
        #     "file_url":  'http://www.sac.net.cn/tzgg/201806/P020180619523158783363.doc',
        #     'file_type': '.doc',
        # },
        # {  # docx 文件不是zip文件
        #     "file_url":'http://www.csrc.gov.cn/pub/zjhpublic/G00306224/zggs/201807/P020180726540550955827.docx',
        #     'file_type': '.docx',
        # },
        {
            "file_url": "http://www.neeq.com.cn/disclosure/2019/2019-04-29/1556524597_292897.pdf",
            'file_type': '.pdf',
        },
    ]
    config = {
        'local': {
            'base_oss_path': 'http://abc-crawler.oss-cn-hangzhou.aliyuncs.com/{}',
            'juchao_mongo_uri': 'mongodb://crawler_team:Ur4Q6h2pgA6X4G9h@114.55.39.144:3717/cr_data',
            'save_file': True,
            'juchao_find_cond': find_cond,
            'bad_case_list': bad_case_list,
            # 'check_oss': True
        },
        'online': {
            'base_oss_path': 'http://abc-crawler.oss-cn-hangzhou-internal.aliyuncs.com/{}',
            'juchao_mongo_uri': 'mongodb://spider2:ThE_eJedRE5a@dds-bp1d09d4b278ceb41.mongodb.rds.aliyuncs.com:3717,dds-bp1d09d4b278ceb42.mongodb.rds.aliyuncs.com:3717/cr_data?replicaSet=mgset-3255385',
            'juchao_find_cond': {},
            'bad_case_list': [],
            # 'check_oss': True,
            # 'save_file': True,
        }
    }
    test(config, env='local')
